# encoding: utf-8
__author__ = 'wwj'
import re
import sys
reload(sys)
sys.setdefaultencoding('utf8')
# sys.path.append("/Users/wwj/Downloads/shangtianjun/girl/")
sys.path.append("/Users/wwj/Downloads/shangtianjun/girl/girl/dbShell/")
sys.path.append( "/var/crawler/spider/crawler/girl/dbShell/")
from dbshell import dbshell
import copy
import re
from zlib import crc32

class main(dbshell):
    orderItem = [
            ['gid', ''],
            ['sid', ''],
            ['title', ''],
            ['movie', ''],
            ['thumb', ''],
            ['url', ''],
    ]
    existgirl = {}
    mongoTable = "movie"
    mysqlTable = "movies"
    def __init__(self,spidername):
        super(main,self).__init__(spidername)
        # self.geturltool("select distinct(url) from movie")
        if not self.redisconn.exists("girlss"):
            sql = "select distinct(urlHash),url,id,sid from girl"
            tmp = self.getdb(sql)
            for x in tmp:
                self.redisconn.hset("girlss",x["url"],(x["id"],x["sid"]))

    def clean(self,y):
        data = self.data
        url = y["girlUrl"].replace("https://www.cityheaven.net", "")
        try:
            data["gid"],data["sid"] = eval(self.redisconn.hget("girlss",url))
        except:
            return "no" + url
        data["thumb"] = re.sub("\?cache.*", "", y.get("thumb")).lstrip("//")
        if y["movie"]:
            data["movie"] = y["movie"].lstrip("//")
        # data["title"] = y["title"]
        data["url"] = y["url"].replace("https://www.cityheaven.net", "")
        data["urlHash"] = str(crc32(data["url"])).replace("-","")
        tmpdata = copy.deepcopy(data)
        self.datas.append(tmpdata)

if __name__ == '__main__':
    obj = main("city")
    obj()
